package com.ustcinfo.study.scala.r1.liyang
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by liyang on 2018/8/4
  *
  **/
object scalaDemo1 {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local").setAppName("scalaDemo1")
    //传sparkcon配置
    val sc = new SparkContext(sparkConf)
    //设置文本路径
    val txtRdd = sc.textFile("src/main/resources/sampleData/programmingGuide")
    txtRdd
      //过滤空格
      .filter(x => x.trim().length > 0)
      //按空格分割
      .map(x => x.split(" "))
      //
      .flatMap(x => x)
      //把k转成k,v结构
      .map(x => (x,1))
      //v值相加
      .reduceByKey(_ + _)
      //k,v交换位置，按v值降序，获取前五十行
      .map(x => (x._2, x._1)).sortByKey(false).take(50)
      //打印
      .foreach(println)
  }

}
